package com.github.rabbithome;

import com.github.rabbithome.processor.FangPageProcessor;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.downloader.PhantomJSDownloader;

/**
 * Created by tomalloc on 17-1-9.
 */
public class FangBootstrap {
    public static void main(String[] args) {
//        String userAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36";
        String url="http://zu.fang.com/house-a01/";
        Site site = Site.me().addHeader("Referer", url).setRetryTimes(3).setSleepTime(2000);
        Spider.create(new FangPageProcessor(site)).addUrl(url).thread(5).run();

    }
}
